{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 策略梯度算法\n",
    "不计算Q，根据reward调整不同动作概率\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Anaconda install\\envs\\Gym\\lib\\site-packages\\gym\\envs\\registration.py:555: UserWarning: \u001b[33mWARN: The environment CartPole-v0 is out of date. You should consider upgrading to version `v1`.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([ 0.02476453, -0.01582927, -0.04678942, -0.04770136], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v0', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        done = terminated or truncated\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAF7CAYAAAD4/3BBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAphElEQVR4nO3df3RU9Z3/8ddMfgyEMBMDJJNIgqgoBAh2AcOsrbVLSkC0ssZz1LIQuxz5yiaeaqzFdK2K3WOs9qw/ugp/7K6450ix9oiuVLAIEtYaEFNSfmkWOLTBkklQmhl+SH7MfL5/eJh1NEImhNzPkOfjnHtO5t73zLzv5ySZ1/ncH+MyxhgBAABYxO10AwAAAF9GQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1nE0oDz33HO65JJLNGTIEJWUlOj99993sh0AAGAJxwLKyy+/rOrqaj388MP6wx/+oClTpqisrExtbW1OtQQAACzhcurLAktKSjR9+nT927/9myQpGo2qoKBAd999tx544AEnWgIAAJZIdeJNOzs71dDQoJqamtg6t9ut0tJS1dfXf6W+o6NDHR0dscfRaFRHjx7ViBEj5HK5BqRnAABwbowxOnbsmPLz8+V2n/kgjiMB5ZNPPlEkElFubm7c+tzcXH300Udfqa+trdWyZcsGqj0AAHAeHTp0SKNHjz5jjSMBJVE1NTWqrq6OPQ6FQiosLNShQ4fk9Xod7AwAAPRWOBxWQUGBhg8fftZaRwLKyJEjlZKSotbW1rj1ra2t8vv9X6n3eDzyeDxfWe/1egkoAAAkmd6cnuHIVTzp6emaOnWqNm7cGFsXjUa1ceNGBQIBJ1oCAAAWcewQT3V1tSoqKjRt2jRdffXVevrpp3XixAn94Ac/cKolAABgCccCyq233qojR47ooYceUjAY1FVXXaX169d/5cRZAAAw+Dh2H5RzEQ6H5fP5FAqFOAcFAIAkkcjnN9/FAwAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgnX4PKI888ohcLlfcMn78+Nj2U6dOqbKyUiNGjFBmZqbKy8vV2tra320AAIAkdl5mUCZOnKiWlpbY8u6778a23XvvvXrjjTf0yiuvqK6uTocPH9bNN998PtoAAABJKvW8vGhqqvx+/1fWh0Ih/cd//IdWrVqlv/u7v5MkvfDCC5owYYK2bt2qGTNmnI92AABAkjkvMyj79u1Tfn6+Lr30Us2fP1/Nzc2SpIaGBnV1dam0tDRWO378eBUWFqq+vv5rX6+jo0PhcDhuAQAAF65+DyglJSVauXKl1q9fr+XLl+vgwYP61re+pWPHjikYDCo9PV1ZWVlxz8nNzVUwGPza16ytrZXP54stBQUF/d02AACwSL8f4pkzZ07s5+LiYpWUlGjMmDH69a9/raFDh/bpNWtqalRdXR17HA6HCSkAAFzAzvtlxllZWbriiiu0f/9++f1+dXZ2qr29Pa6mtbW1x3NWTvN4PPJ6vXELAAC4cJ33gHL8+HEdOHBAeXl5mjp1qtLS0rRx48bY9qamJjU3NysQCJzvVgAAQJLo90M8P/rRj3TjjTdqzJgxOnz4sB5++GGlpKTo9ttvl8/n06JFi1RdXa3s7Gx5vV7dfffdCgQCXMEDAABi+j2gfPzxx7r99tv16aefatSoUfrmN7+prVu3atSoUZKkp556Sm63W+Xl5ero6FBZWZmef/75/m4DAAAkMZcxxjjdRKLC4bB8Pp9CoRDnowAAkCQS+fzmu3gAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANZJOKBs2bJFN954o/Lz8+VyufTaa6/FbTfG6KGHHlJeXp6GDh2q0tJS7du3L67m6NGjmj9/vrxer7KysrRo0SIdP378nHYEAABcOBIOKCdOnNCUKVP03HPP9bj9iSee0LPPPqsVK1Zo27ZtGjZsmMrKynTq1KlYzfz587Vnzx5t2LBBa9eu1ZYtW7R48eK+7wUAALiguIwxps9Pdrm0Zs0azZs3T9Lnsyf5+fm677779KMf/UiSFAqFlJubq5UrV+q2227Thx9+qKKiIm3fvl3Tpk2TJK1fv17XX3+9Pv74Y+Xn55/1fcPhsHw+n0KhkLxeb1/bBwAAAyiRz+9+PQfl4MGDCgaDKi0tja3z+XwqKSlRfX29JKm+vl5ZWVmxcCJJpaWlcrvd2rZtW4+v29HRoXA4HLcAAIALV78GlGAwKEnKzc2NW5+bmxvbFgwGlZOTE7c9NTVV2dnZsZovq62tlc/niy0FBQX92TYAALBMUlzFU1NTo1AoFFsOHTrkdEsAAOA86teA4vf7JUmtra1x61tbW2Pb/H6/2tra4rZ3d3fr6NGjsZov83g88nq9cQsAALhw9WtAGTt2rPx+vzZu3BhbFw6HtW3bNgUCAUlSIBBQe3u7GhoaYjWbNm1SNBpVSUlJf7YDAACSVGqiTzh+/Lj2798fe3zw4EE1NjYqOztbhYWFuueee/Qv//IvGjdunMaOHauf/vSnys/Pj13pM2HCBM2ePVt33nmnVqxYoa6uLlVVVem2227r1RU8AADgwpdwQPnggw/0ne98J/a4urpaklRRUaGVK1fqxz/+sU6cOKHFixervb1d3/zmN7V+/XoNGTIk9pyXXnpJVVVVmjlzptxut8rLy/Xss8/2w+4AAIALwTndB8Up3AcFAIDk49h9UAAAAPoDAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUSDihbtmzRjTfeqPz8fLlcLr322mtx2++44w65XK64Zfbs2XE1R48e1fz58+X1epWVlaVFixbp+PHj57QjAADgwpFwQDlx4oSmTJmi55577mtrZs+erZaWltjyq1/9Km77/PnztWfPHm3YsEFr167Vli1btHjx4sS7BwAAF6TURJ8wZ84czZkz54w1Ho9Hfr+/x20ffvih1q9fr+3bt2vatGmSpF/+8pe6/vrr9Ytf/EL5+fmJtgQAAC4w5+UclM2bNysnJ0dXXnmllixZok8//TS2rb6+XllZWbFwIkmlpaVyu93atm1bj6/X0dGhcDgctwAAgAtXvweU2bNn67/+67+0ceNG/fznP1ddXZ3mzJmjSCQiSQoGg8rJyYl7TmpqqrKzsxUMBnt8zdraWvl8vthSUFDQ320DAACLJHyI52xuu+222M+TJ09WcXGxLrvsMm3evFkzZ87s02vW1NSouro69jgcDhNSAAC4gJ33y4wvvfRSjRw5Uvv375ck+f1+tbW1xdV0d3fr6NGjX3veisfjkdfrjVsAAMCF67wHlI8//liffvqp8vLyJEmBQEDt7e1qaGiI1WzatEnRaFQlJSXnux0AAJAEEj7Ec/z48dhsiCQdPHhQjY2Nys7OVnZ2tpYtW6by8nL5/X4dOHBAP/7xj3X55ZerrKxMkjRhwgTNnj1bd955p1asWKGuri5VVVXptttu4woeAAAgSXIZY0wiT9i8ebO+853vfGV9RUWFli9frnnz5mnHjh1qb29Xfn6+Zs2apZ/97GfKzc2N1R49elRVVVV644035Ha7VV5ermeffVaZmZm96iEcDsvn8ykUCnG4BwCAJJHI53fCAcUGBBQAAJJPIp/ffBcPAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFgn4S8LBID+EPzj7xQ+3HTGmpFXBJR92bQB6giATQgoABxx8tNDCjXvOmPNsFGXyESjcrmZ7AUGG/7qAVjLRLplTMTpNgA4gIACwFrRSJdMNOp0GwAcQEABYK3PAwozKMBgREABYK1opFvGMIMCDEYEFADWMpEuiRkUYFAioACwVrSbc1CAwYqAAsBaJtLFIR5gkCKgAHBExsgxcqcNOWPNyaN/UdfJ8AB1BMAmBBQAjhh6UZ5S0jxnrOk68VdFOj8boI4A2ISAAsAR7tR0yeVyug0AliKgAHCEOzVdLhf/ggD0jP8OABzhSkljBgXA1yKgAHAEMygAzoT/DgAckcI5KADOgIACwBGu1DRmUAB8Lf47AHBE72dQjIwx570fAHYhoABwhMudIpfOHlC41T0wOBFQAFgt0t0hiRkUYLBJKKDU1tZq+vTpGj58uHJycjRv3jw1NTXF1Zw6dUqVlZUaMWKEMjMzVV5ertbW1ria5uZmzZ07VxkZGcrJydH999+v7u7uc98bABecaFenxCEeYNBJKKDU1dWpsrJSW7du1YYNG9TV1aVZs2bpxIkTsZp7771Xb7zxhl555RXV1dXp8OHDuvnmm2PbI5GI5s6dq87OTr333nt68cUXtXLlSj300EP9t1cALhjR7g7OQQEGIZc5h7/8I0eOKCcnR3V1dbr22msVCoU0atQorVq1Srfccosk6aOPPtKECRNUX1+vGTNmaN26dbrhhht0+PBh5ebmSpJWrFihpUuX6siRI0pPTz/r+4bDYfl8PoVCIXm93r62D8Bhu3/9iD776+Ez1owuuVm5k0vlTkkdoK4AnC+JfH6f0zkooVBIkpSdnS1JamhoUFdXl0pLS2M148ePV2Fhoerr6yVJ9fX1mjx5ciycSFJZWZnC4bD27NnT4/t0dHQoHA7HLQAGhwiHeIBBqc8BJRqN6p577tE111yjSZMmSZKCwaDS09OVlZUVV5ubm6tgMBir+WI4Ob399Lae1NbWyufzxZaCgoK+tg0gyUQ5SRYYlPocUCorK7V7926tXr26P/vpUU1NjUKhUGw5dOjQeX9PAHaIdndyDgowCPXpoG5VVZXWrl2rLVu2aPTo0bH1fr9fnZ2dam9vj5tFaW1tld/vj9W8//77ca93+iqf0zVf5vF45PF4+tIqAIuleIadtabrZEgy3AsFGGwSmkExxqiqqkpr1qzRpk2bNHbs2LjtU6dOVVpamjZu3Bhb19TUpObmZgUCAUlSIBDQrl271NbWFqvZsGGDvF6vioqKzmVfACSZUeOvkc5yu/v2PzUq0t05QB0BsEVCMyiVlZVatWqVXn/9dQ0fPjx2zojP59PQoUPl8/m0aNEiVVdXKzs7W16vV3fffbcCgYBmzJghSZo1a5aKioq0YMECPfHEEwoGg3rwwQdVWVnJLAkwyLjThsglzjAB8FUJBZTly5dLkq677rq49S+88ILuuOMOSdJTTz0lt9ut8vJydXR0qKysTM8//3ysNiUlRWvXrtWSJUsUCAQ0bNgwVVRU6NFHHz23PQGQdFLSPJ9/Hw8JBcCXnNN9UJzCfVCAC8Ox4AE1vfELmWjkjHVTFjyp9AzfAHUF4HwZsPugAMC5SEnjsC6AnhFQADjGnTZE6sU3GgMYfAgoABzDDAqAr0NAAeAYd5qHCRQAPSKgAHCMOyWtV3UmGuVussAgQ0ABYL1o1ymnWwAwwAgoAKwXIaAAgw4BBYD1mEEBBh8CCgDrRbo6nG4BwAAjoACwXqSTGRRgsCGgALAe56AAgw8BBYD1ohziAQYdAgoA6zGDAgw+BBQAjsqZ8O2z1nzy0bsD0AkAmxBQADgqLTP7rDXdHScHoBMANiGgAHAUXxgIoCcEFACOSkkb4nQLACxEQAHgKHc6MygAvoqAAsBRzKAA6AkBBYCj3AQUAD0goABwVCoBBUAPCCgAHOVOTe9VnTHR89wJAJsQUAA4xuVySS5Xr2qj3Z3nuRsANiGgAEgCRtFuvo8HGEwIKACSQrSTgAIMJgQUAPYzUoQZFGBQIaAASAKGbzQGBhkCCoCkEO1iBgUYTAgoAJJChIACDCoEFACOcrncSh0y/Iw1xhh1hNoGqCMANiCgAHCUOzVdWWMmn7nIRHX0wPaBaQiAFRIKKLW1tZo+fbqGDx+unJwczZs3T01NTXE11113nVwuV9xy1113xdU0Nzdr7ty5ysjIUE5Oju6//351d3ef+94ASD4ul9ypfKMxgHipiRTX1dWpsrJS06dPV3d3t37yk59o1qxZ2rt3r4YNGxaru/POO/Xoo4/GHmdkZMR+jkQimjt3rvx+v9577z21tLRo4cKFSktL02OPPdYPuwQgmbgIKAB6kFBAWb9+fdzjlStXKicnRw0NDbr22mtj6zMyMuT3+3t8jd/97nfau3ev3n77beXm5uqqq67Sz372My1dulSPPPKI0tN7970cAC4ULrnT+LsHEO+czkEJhUKSpOzs7Lj1L730kkaOHKlJkyappqZGJ0+ejG2rr6/X5MmTlZubG1tXVlamcDisPXv29Pg+HR0dCofDcQuAC4TL1esvDAQweCQ0g/JF0WhU99xzj6655hpNmjQptv773/++xowZo/z8fO3cuVNLly5VU1OTXn31VUlSMBiMCyeSYo+DwWCP71VbW6tly5b1tVUANnO5lMIMCoAv6XNAqays1O7du/Xuu+/GrV+8eHHs58mTJysvL08zZ87UgQMHdNlll/XpvWpqalRdXR17HA6HVVBQ0LfGAVjFJc5BAfBVfTrEU1VVpbVr1+qdd97R6NGjz1hbUlIiSdq/f78kye/3q7W1Na7m9OOvO2/F4/HI6/XGLQAuEC5xiAfAVyQUUIwxqqqq0po1a7Rp0yaNHTv2rM9pbGyUJOXl5UmSAoGAdu3apba2/7vp0oYNG+T1elVUVJRIOwAuCL2fQTHGnOdeANgioUM8lZWVWrVqlV5//XUNHz48ds6Iz+fT0KFDdeDAAa1atUrXX3+9RowYoZ07d+ree+/Vtddeq+LiYknSrFmzVFRUpAULFuiJJ55QMBjUgw8+qMrKSnk8TPMCg83p+yWdlTEy0W65UtLOf1MAHJfQDMry5csVCoV03XXXKS8vL7a8/PLLkqT09HS9/fbbmjVrlsaPH6/77rtP5eXleuONN2KvkZKSorVr1yolJUWBQED/8A//oIULF8bdNwUAvswYo2h3l9NtABggCc2gnG16taCgQHV1dWd9nTFjxujNN99M5K0BDHLGRBXt7pQ8GWcvBpD0+C4eAMnhdEABMCgQUAAkBQ7xAIMLAQVAcjBRRSPMoACDBQEFQFIwHOIBBhUCCoDkwCEeYFAhoABwnMc7Upl5V5yxputkWO0HdwxQRwCcRkAB4DiXO1UpaWe7UaNRNNo9IP0AcB4BBYDjXO4UublDLIAvIKAAcJzL7eYW9gDiEFAAOM7lSpE7JaEbWwO4wBFQADjP7ZaLgALgCwgoABzncqfIncohHgD/h4ACwHEul1tuNwEFwP8hoABwnMudIhczKAC+gIACwHEul0sul+vshdGoTDRy/hsC4DgCCoCkEY1GFCWgAIMCAQVA0jDRbpkId5MFBgMCCoCkYaIRAgowSBBQACQNE4nwfTzAIMGdkQD0C2OMIpG+nx8SjUbPXhPpUndnp1K6zy2kpKSk9O6kXACOIaAA6Bf79u3TxIkT+/z8irJi/b8bp56x5r3fv6sbK3+uP7eG+vw+Ho9H4XCYgAJYjoACoF8YY9R9DjMbvZlBSXG75NK5vU9KSkqfnwtg4BBQAFjhL58c09HwZ8r2DlW3SVVrxyX6LDpcLhllpvxVOel/Vm52pi4eOVz7/3LU6XYBnGcEFABW+LgtrKPhz3TR8Az9ITxLx7pHqNN45JJRuvszHekq0CTfu8q5aJjTrQIYAAQUAFbo7I6oMyJtDX1P7d05kj4/R8RI6ohm6uNT4+VWVFK9k20CGCBcZgzACl3dEX3Q/p24cPJFRm79+dRENZ8qGvjmAAw4AgoAK3R2R9QdMeopnPwfl8xANQTAUQQUAFbo7Ioo0osreQAMDgQUAFbo6o6oO0JAAfA5AgoAK3R2RzUxY5MyU/4q9Xggx+hiT5MKhnw00K0BcEBCAWX58uUqLi6W1+uV1+tVIBDQunXrYttPnTqlyspKjRgxQpmZmSovL1dra2vcazQ3N2vu3LnKyMhQTk6O7r///nO66RKAC0NXd0SKduibWb+RN+UTpbo6JEXlUlRprs+Ul35AkzPr5Bb/L4DBIKHLjEePHq3HH39c48aNkzFGL774om666Sbt2LFDEydO1L333qvf/va3euWVV+Tz+VRVVaWbb75Zv//97yVJkUhEc+fOld/v13vvvaeWlhYtXLhQaWlpeuyxx87LDgJIDpGoUf3eQzp67DN1m/36y6lxOhG5SC5F5U39RMeH7NOfJB1saXe4UwADwWWMOaeT4rOzs/Xkk0/qlltu0ahRo7Rq1SrdcsstkqSPPvpIEyZMUH19vWbMmKF169bphhtu0OHDh5WbmytJWrFihZYuXaojR44oPT29V+8ZDofl8/l0xx139Po5AM6vUCikl19+2ek2zsrtdmvRokV8Fw/ggM7OTq1cuVKhUEher/eMtX2+UVskEtErr7yiEydOKBAIqKGhQV1dXSotLY3VjB8/XoWFhbGAUl9fr8mTJ8fCiSSVlZVpyZIl2rNnj77xjW/0+F4dHR3q6OiIPQ6Hw5KkBQsWKDMzs6+7AKAfNTc3J0VASUlJIaAADjl+/LhWrlzZq9qEA8quXbsUCAR06tQpZWZmas2aNSoqKlJjY6PS09OVlZUVV5+bm6tgMChJCgaDceHk9PbT275ObW2tli1b9pX106ZNO2sCAzAwfD6f0y30itvt1vTp0+V2c40AMNBOTzD0RsJ/oVdeeaUaGxu1bds2LVmyRBUVFdq7d2+iL5OQmpoahUKh2HLo0KHz+n4AAMBZCc+gpKen6/LLL5ckTZ06Vdu3b9czzzyjW2+9VZ2dnWpvb4+bRWltbZXf75ck+f1+vf/++3Gvd/oqn9M1PfF4PPJ4PIm2CgAAktQ5z3FGo1F1dHRo6tSpSktL08aNG2Pbmpqa1NzcrEAgIEkKBALatWuX2traYjUbNmyQ1+tVURHfrwEAAD6X0AxKTU2N5syZo8LCQh07dkyrVq3S5s2b9dZbb8nn82nRokWqrq5Wdna2vF6v7r77bgUCAc2YMUOSNGvWLBUVFWnBggV64oknFAwG9eCDD6qyspIZEgAAEJNQQGlra9PChQvV0tIin8+n4uJivfXWW/rud78rSXrqqafkdrtVXl6ujo4OlZWV6fnnn489PyUlRWvXrtWSJUsUCAQ0bNgwVVRU6NFHH+3fvQIAAEntnO+D4oTT90HpzXXUAAZGU1OTxo8f73QbZ+XxeHTy5Emu4gEckMjnN3+hAADAOgQUAABgHQIKAACwDgEFAABYp8/fxQMAX5SZmal58+Y53cZZpaWlOd0CgF4goADoFxdffLHWrFnjdBsALhAc4gEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKyTUEBZvny5iouL5fV65fV6FQgEtG7dutj26667Ti6XK26566674l6jublZc+fOVUZGhnJycnT//feru7u7f/YGAABcEFITKR49erQef/xxjRs3TsYYvfjii7rpppu0Y8cOTZw4UZJ055136tFHH409JyMjI/ZzJBLR3Llz5ff79d5776mlpUULFy5UWlqaHnvssX7aJQAAkOxcxhhzLi+QnZ2tJ598UosWLdJ1112nq666Sk8//XSPtevWrdMNN9ygw4cPKzc3V5K0YsUKLV26VEeOHFF6enqv3jMcDsvn8ykUCsnr9Z5L+wAAYIAk8vnd53NQIpGIVq9erRMnTigQCMTWv/TSSxo5cqQmTZqkmpoanTx5Mratvr5ekydPjoUTSSorK1M4HNaePXu+9r06OjoUDofjFgAAcOFK6BCPJO3atUuBQECnTp1SZmam1qxZo6KiIknS97//fY0ZM0b5+fnauXOnli5dqqamJr366quSpGAwGBdOJMUeB4PBr33P2tpaLVu2LNFWAQBAkko4oFx55ZVqbGxUKBTSb37zG1VUVKiurk5FRUVavHhxrG7y5MnKy8vTzJkzdeDAAV122WV9brKmpkbV1dWxx+FwWAUFBX1+PQAAYLeED/Gkp6fr8ssv19SpU1VbW6spU6bomWee6bG2pKREkrR//35Jkt/vV2tra1zN6cd+v/9r39Pj8cSuHDq9AACAC9c53wclGo2qo6Ojx22NjY2SpLy8PElSIBDQrl271NbWFqvZsGGDvF5v7DARAABAQod4ampqNGfOHBUWFurYsWNatWqVNm/erLfeeksHDhzQqlWrdP3112vEiBHauXOn7r33Xl177bUqLi6WJM2aNUtFRUVasGCBnnjiCQWDQT344IOqrKyUx+M5LzsIAACST0IBpa2tTQsXLlRLS4t8Pp+Ki4v11ltv6bvf/a4OHTqkt99+W08//bROnDihgoIClZeX68EHH4w9PyUlRWvXrtWSJUsUCAQ0bNgwVVRUxN03BQAA4Jzvg+IE7oMCAEDyGZD7oAAAAJwvBBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDoEFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABYh4ACAACsQ0ABAADWIaAAAADrEFAAAIB1CCgAAMA6BBQAAGAdAgoAALAOAQUAAFiHgAIAAKxDQAEAANYhoAAAAOsQUAAAgHUIKAAAwDqpTjfQF8YYSVI4HHa4EwAA0FunP7dPf46fSVIGlGPHjkmSCgoKHO4EAAAk6tixY/L5fGescZnexBjLRKNRNTU1qaioSIcOHZLX63W6paQVDodVUFDAOPYDxrL/MJb9g3HsP4xl/zDG6NixY8rPz5fbfeazTJJyBsXtduviiy+WJHm9Xn5Z+gHj2H8Yy/7DWPYPxrH/MJbn7mwzJ6dxkiwAALAOAQUAAFgnaQOKx+PRww8/LI/H43QrSY1x7D+MZf9hLPsH49h/GMuBl5QnyQIAgAtb0s6gAACACxcBBQAAWIeAAgAArENAAQAA1knKgPLcc8/pkksu0ZAhQ1RSUqL333/f6Zass2XLFt14443Kz8+Xy+XSa6+9FrfdGKOHHnpIeXl5Gjp0qEpLS7Vv3764mqNHj2r+/Pnyer3KysrSokWLdPz48QHcC+fV1tZq+vTpGj58uHJycjRv3jw1NTXF1Zw6dUqVlZUaMWKEMjMzVV5ertbW1ria5uZmzZ07VxkZGcrJydH999+v7u7ugdwVRy1fvlzFxcWxm1wFAgGtW7cutp0x7LvHH39cLpdL99xzT2wd49k7jzzyiFwuV9wyfvz42HbG0WEmyaxevdqkp6eb//zP/zR79uwxd955p8nKyjKtra1Ot2aVN9980/zzP/+zefXVV40ks2bNmrjtjz/+uPH5fOa1114zf/zjH833vvc9M3bsWPPZZ5/FambPnm2mTJlitm7dav7nf/7HXH755eb2228f4D1xVllZmXnhhRfM7t27TWNjo7n++utNYWGhOX78eKzmrrvuMgUFBWbjxo3mgw8+MDNmzDB/+7d/G9ve3d1tJk2aZEpLS82OHTvMm2++aUaOHGlqamqc2CVH/Pd//7f57W9/a/73f//XNDU1mZ/85CcmLS3N7N692xjDGPbV+++/by655BJTXFxsfvjDH8bWM5698/DDD5uJEyealpaW2HLkyJHYdsbRWUkXUK6++mpTWVkZexyJREx+fr6pra11sCu7fTmgRKNR4/f7zZNPPhlb197ebjwej/nVr35ljDFm7969RpLZvn17rGbdunXG5XKZv/zlLwPWu23a2tqMJFNXV2eM+Xzc0tLSzCuvvBKr+fDDD40kU19fb4z5PCy63W4TDAZjNcuXLzder9d0dHQM7A5Y5KKLLjL//u//zhj20bFjx8y4cePMhg0bzLe//e1YQGE8e+/hhx82U6ZM6XEb4+i8pDrE09nZqYaGBpWWlsbWud1ulZaWqr6+3sHOksvBgwcVDAbjxtHn86mkpCQ2jvX19crKytK0adNiNaWlpXK73dq2bduA92yLUCgkScrOzpYkNTQ0qKurK24sx48fr8LCwrixnDx5snJzc2M1ZWVlCofD2rNnzwB2b4dIJKLVq1frxIkTCgQCjGEfVVZWau7cuXHjJvE7mah9+/YpPz9fl156qebPn6/m5mZJjKMNkurLAj/55BNFIpG4XwZJys3N1UcffeRQV8knGAxKUo/jeHpbMBhUTk5O3PbU1FRlZ2fHagabaDSqe+65R9dcc40mTZok6fNxSk9PV1ZWVlztl8eyp7E+vW2w2LVrlwKBgE6dOqXMzEytWbNGRUVFamxsZAwTtHr1av3hD3/Q9u3bv7KN38neKykp0cqVK3XllVeqpaVFy5Yt07e+9S3t3r2bcbRAUgUUwEmVlZXavXu33n33XadbSUpXXnmlGhsbFQqF9Jvf/EYVFRWqq6tzuq2kc+jQIf3whz/Uhg0bNGTIEKfbSWpz5syJ/VxcXKySkhKNGTNGv/71rzV06FAHO4OUZFfxjBw5UikpKV85i7q1tVV+v9+hrpLP6bE60zj6/X61tbXFbe/u7tbRo0cH5VhXVVVp7dq1eueddzR69OjYer/fr87OTrW3t8fVf3ksexrr09sGi/T0dF1++eWaOnWqamtrNWXKFD3zzDOMYYIaGhrU1tamv/mbv1FqaqpSU1NVV1enZ599VqmpqcrNzWU8+ygrK0tXXHGF9u/fz++lBZIqoKSnp2vq1KnauHFjbF00GtXGjRsVCAQc7Cy5jB07Vn6/P24cw+Gwtm3bFhvHQCCg9vZ2NTQ0xGo2bdqkaDSqkpKSAe/ZKcYYVVVVac2aNdq0aZPGjh0bt33q1KlKS0uLG8umpiY1NzfHjeWuXbviAt+GDRvk9XpVVFQ0MDtioWg0qo6ODsYwQTNnztSuXbvU2NgYW6ZNm6b58+fHfmY8++b48eM6cOCA8vLy+L20gdNn6SZq9erVxuPxmJUrV5q9e/eaxYsXm6ysrLizqPH5Gf47duwwO3bsMJLMv/7rv5odO3aYP//5z8aYzy8zzsrKMq+//rrZuXOnuemmm3q8zPgb3/iG2bZtm3n33XfNuHHjBt1lxkuWLDE+n89s3rw57lLEkydPxmruuusuU1hYaDZt2mQ++OADEwgETCAQiG0/fSnirFmzTGNjo1m/fr0ZNWrUoLoU8YEHHjB1dXXm4MGDZufOneaBBx4wLpfL/O53vzPGMIbn6otX8RjDePbWfffdZzZv3mwOHjxofv/735vS0lIzcuRI09bWZoxhHJ2WdAHFGGN++ctfmsLCQpOenm6uvvpqs3XrVqdbss4777xjJH1lqaioMMZ8fqnxT3/6U5Obm2s8Ho+ZOXOmaWpqinuNTz/91Nx+++0mMzPTeL1e84Mf/MAcO3bMgb1xTk9jKMm88MILsZrPPvvM/NM//ZO56KKLTEZGhvn7v/9709LSEvc6f/rTn8ycOXPM0KFDzciRI819991nurq6BnhvnPOP//iPZsyYMSY9Pd2MGjXKzJw5MxZOjGEMz9WXAwrj2Tu33nqrycvLM+np6ebiiy82t956q9m/f39sO+PoLJcxxjgzdwMAANCzpDoHBQAADA4EFAAAYB0CCgAAsA4BBQAAWIeAAgAArENAAQAA1iGgAAAA6xBQAACAdQgoAADAOgQUAABgHQIKAACwDgEFAABY5/8DM4seyatM9wcAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.5214, 0.4786],\n",
       "        [0.4621, 0.5379]], grad_fn=<SoftmaxBackward0>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#定义模型\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4,128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128,2),\n",
    "    torch.nn.Softmax(dim=1)#因为最后要输出每个动作的概率   \n",
    ")\n",
    "model(torch.randn(2,4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "# 得到一个动作\n",
    "def get_action(state):\n",
    "    state = torch.FloatTensor(state).reshape(1, 4)\n",
    "    prob = model(state)\n",
    "    # 根据概率选择一个动作\n",
    "    action = random.choices(range(2), weights=prob[0].tolist(), k=1)[0]  # 根据概率对动作采样一次\n",
    "    return action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_action(env.reset())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([array([-0.030349  ,  0.00617239, -0.02154703, -0.03570786], dtype=float32),\n",
       "  array([-0.03022555,  0.2015966 , -0.02226118, -0.33511046], dtype=float32),\n",
       "  array([-0.02619362,  0.39702818, -0.02896339, -0.63472944], dtype=float32),\n",
       "  array([-0.01825306,  0.5925419 , -0.04165798, -0.93639106], dtype=float32),\n",
       "  array([-0.00640222,  0.78820014, -0.0603858 , -1.2418677 ], dtype=float32),\n",
       "  array([ 0.00936178,  0.984043  , -0.08522315, -1.5528393 ], dtype=float32),\n",
       "  array([ 0.02904264,  1.1800771 , -0.11627994, -1.8708484 ], dtype=float32),\n",
       "  array([ 0.05264419,  1.3762623 , -0.15369691, -2.1972482 ], dtype=float32),\n",
       "  array([ 0.08016943,  1.1829206 , -0.19764188, -1.9556636 ], dtype=float32)],\n",
       " [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],\n",
       " [1, 1, 1, 1, 1, 1, 1, 0, 1])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 得到一句游戏的数据\n",
    "def get_data():\n",
    "    states=[]\n",
    "    rewards=[]\n",
    "    actions=[]\n",
    "\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作，得到反馈\n",
    "        next_state,reward,over,_=env.step(action)\n",
    "\n",
    "        #记录数据样本\n",
    "        states.append(state)\n",
    "        rewards.append(reward)\n",
    "        actions.append(action)\n",
    "\n",
    "        #更新游戏状态，开始下一个动作\n",
    "        state = next_state\n",
    "    return states,rewards,actions\n",
    "get_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "66.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    \n",
    "    while not over:\n",
    "        \n",
    "        #根据当前状态得到一个动作\n",
    "        action = get_action(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play: \n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:0,score:22.2\n",
      "epoch:100,score:55.4\n",
      "epoch:200,score:102.3\n",
      "epoch:300,score:156.9\n",
      "epoch:400,score:175.9\n",
      "epoch:500,score:169.7\n",
      "epoch:600,score:138.4\n",
      "epoch:700,score:187.7\n",
      "epoch:800,score:197.6\n",
      "epoch:900,score:200.0\n"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n",
    "\n",
    "    # 玩N局游戏，每局游戏训练一次\n",
    "    for epoch in range(1000):\n",
    "        states, rewards, actions = get_data()\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        rewards_sum = 0\n",
    "\n",
    "        # 从最后一步算起\n",
    "        for i in reversed(range(len(states))):\n",
    "            # 反馈的和，从最后一步的反馈开始计算，\n",
    "            # 每往前一步，和都衰减0.02，然后再加上当前步的反馈\n",
    "            rewards_sum *= 0.98\n",
    "            rewards_sum += rewards[i]\n",
    "\n",
    "            # 重新计算对应动作的概率\n",
    "            state = torch.FloatTensor(states[i]).reshape(1, 4)\n",
    "\n",
    "            prob = model(state)\n",
    "\n",
    "            prob = prob[0, actions[i]]  # 将对应动作的那个概率取出来\n",
    "\n",
    "            loss = -prob.log() * rewards_sum\n",
    "\n",
    "            # 累积梯度\n",
    "            loss.backward(retain_graph=True)\n",
    "        optimizer.step()\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(10)]) / 10\n",
    "            print(f\"epoch:{epoch},score:{test_result}\")\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAF7CAYAAAD4/3BBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAo2klEQVR4nO3df3RU9Z3/8ddMfgyEMBMDJJOUBFEoECHoAoap1qVLSoDoyhrPUctCbDlwoImnEEsxXSpi9xiLe9ZfRfhjW3HPkdLSI7pSwcYgYa3hh5EsvyQVlm2wZBIqm5kQm5/z+f7Blzk7EgKTTDI3+Hycc8/J3M9n7rzv5+SQF5/7uXdsxhgjAAAAC7FHuwAAAIAvI6AAAADLIaAAAADLIaAAAADLIaAAAADLIaAAAADLIaAAAADLIaAAAADLIaAAAADLIaAAAADLiWpA2bhxo26++WYNGTJEOTk5OnjwYDTLAQAAFhG1gPLrX/9aJSUlWrdunT7++GNNnTpVeXl5amxsjFZJAADAImzR+rLAnJwczZgxQz//+c8lSYFAQBkZGXrsscf0xBNPRKMkAABgEbHR+ND29nZVV1ertLQ0uM9utys3N1dVVVVX9G9ra1NbW1vwdSAQ0IULFzRixAjZbLYBqRkAAPSNMUbNzc1KT0+X3d7zRZyoBJS//OUv6urqUmpqasj+1NRUnTx58or+ZWVlWr9+/UCVBwAA+tHZs2c1evToHvtEJaCEq7S0VCUlJcHXPp9PmZmZOnv2rJxOZxQrAwAA18vv9ysjI0PDhw+/Zt+oBJSRI0cqJiZGDQ0NIfsbGhrkdruv6O9wOORwOK7Y73Q6CSgAAAwy17M8Iyp38cTHx2vatGmqqKgI7gsEAqqoqJDH44lGSQAAwEKidomnpKREhYWFmj59uu6880698MILamlp0Xe/+91olQQAACwiagHloYce0vnz5/Xkk0/K6/Xq9ttv1+7du69YOAsAAL56ovYclL7w+/1yuVzy+XysQQEAYJAI5+8338UDAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsh4ACAAAsJ+IB5amnnpLNZgvZJk6cGGxvbW1VUVGRRowYocTERBUUFKihoSHSZQAAgEGsX2ZQbrvtNtXX1we3Dz74INi2atUqvf3229q+fbsqKyt17tw5PfDAA/1RBgAAGKRi++WgsbFyu91X7Pf5fPrFL36hrVu36u/+7u8kSa+++qomTZqk/fv3a+bMmf1RDgAAGGT6ZQbl008/VXp6um655RYtXLhQdXV1kqTq6mp1dHQoNzc32HfixInKzMxUVVXVVY/X1tYmv98fsgEAgBtXxANKTk6OtmzZot27d2vTpk06c+aMvvnNb6q5uVler1fx8fFKSkoKeU9qaqq8Xu9Vj1lWViaXyxXcMjIyIl02AACwkIhf4pk3b17w5+zsbOXk5GjMmDH6zW9+o6FDh/bqmKWlpSopKQm+9vv9hBQAAG5g/X6bcVJSkr7+9a/r1KlTcrvdam9vV1NTU0ifhoaGbtesXOZwOOR0OkM2AABw4+r3gHLx4kWdPn1aaWlpmjZtmuLi4lRRURFsr62tVV1dnTweT3+XAgAABomIX+L54Q9/qPvuu09jxozRuXPntG7dOsXExOiRRx6Ry+XSkiVLVFJSouTkZDmdTj322GPyeDzcwQMAAIIiHlA+++wzPfLII/r88881atQo3X333dq/f79GjRolSXr++edlt9tVUFCgtrY25eXl6ZVXXol0GQAAYBCzGWNMtIsIl9/vl8vlks/nYz0KAACDRDh/v/kuHgAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDkEFAAAYDlhB5R9+/bpvvvuU3p6umw2m958882QdmOMnnzySaWlpWno0KHKzc3Vp59+GtLnwoULWrhwoZxOp5KSkrRkyRJdvHixTycCAABuHGEHlJaWFk2dOlUbN27stn3Dhg166aWXtHnzZh04cEDDhg1TXl6eWltbg30WLlyo48ePq7y8XDt37tS+ffu0bNmy3p8FAAC4odiMMabXb7bZtGPHDi1YsEDSpdmT9PR0Pf744/rhD38oSfL5fEpNTdWWLVv08MMP65NPPlFWVpYOHTqk6dOnS5J2796t+fPn67PPPlN6evo1P9fv98vlcsnn88npdPa2fAAAMIDC+fsd0TUoZ86ckdfrVW5ubnCfy+VSTk6OqqqqJElVVVVKSkoKhhNJys3Nld1u14EDB7o9bltbm/x+f8gGAABuXBENKF6vV5KUmpoasj81NTXY5vV6lZKSEtIeGxur5OTkYJ8vKysrk8vlCm4ZGRmRLBsAAFjMoLiLp7S0VD6fL7idPXs22iUBAIB+FNGA4na7JUkNDQ0h+xsaGoJtbrdbjY2NIe2dnZ26cOFCsM+XORwOOZ3OkA0AANy4IhpQxo4dK7fbrYqKiuA+v9+vAwcOyOPxSJI8Ho+amppUXV0d7LNnzx4FAgHl5OREshwAADBIxYb7hosXL+rUqVPB12fOnFFNTY2Sk5OVmZmplStX6p//+Z81fvx4jR07Vj/5yU+Unp4evNNn0qRJmjt3rpYuXarNmzero6NDxcXFevjhh6/rDh4AAHDjCzugfPTRR/rWt74VfF1SUiJJKiws1JYtW/SjH/1ILS0tWrZsmZqamnT33Xdr9+7dGjJkSPA9r7/+uoqLizV79mzZ7XYVFBTopZdeisDpAACAG0GfnoMSLTwHBQCAwSdqz0EBAACIBAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwnLADyr59+3TfffcpPT1dNptNb775Zkj7o48+KpvNFrLNnTs3pM+FCxe0cOFCOZ1OJSUlacmSJbp48WKfTgQAANw4wg4oLS0tmjp1qjZu3HjVPnPnzlV9fX1w+9WvfhXSvnDhQh0/flzl5eXauXOn9u3bp2XLloVfPQAAuCHFhvuGefPmad68eT32cTgccrvd3bZ98skn2r17tw4dOqTp06dLkl5++WXNnz9f//Iv/6L09PRwSwIAADeYflmDsnfvXqWkpGjChAlasWKFPv/882BbVVWVkpKSguFEknJzc2W323XgwIFuj9fW1ia/3x+yAQCAG1fEA8rcuXP17//+76qoqNDPfvYzVVZWat68eerq6pIkeb1epaSkhLwnNjZWycnJ8nq93R6zrKxMLpcruGVkZES6bAAAYCFhX+K5locffjj485QpU5Sdna1bb71Ve/fu1ezZs3t1zNLSUpWUlARf+/1+QgoAADewfr/N+JZbbtHIkSN16tQpSZLb7VZjY2NIn87OTl24cOGq61YcDoecTmfIBgAAblz9HlA+++wzff7550pLS5MkeTweNTU1qbq6Othnz549CgQCysnJ6e9yAADAIBD2JZ6LFy8GZ0Mk6cyZM6qpqVFycrKSk5O1fv16FRQUyO126/Tp0/rRj36kcePGKS8vT5I0adIkzZ07V0uXLtXmzZvV0dGh4uJiPfzww9zBAwAAJEk2Y4wJ5w179+7Vt771rSv2FxYWatOmTVqwYIEOHz6spqYmpaena86cOfrpT3+q1NTUYN8LFy6ouLhYb7/9tux2uwoKCvTSSy8pMTHxumrw+/1yuVzy+Xxc7gEAYJAI5+932AHFCggoAAAMPuH8/ea7eAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOUQUAAAgOWE/WWBANCT8yc/0P+eOdxjn6QxU5WSdc8AVQRgMCKgAIio1v+tl6/uaI99HMNHypiAbDYmcQF0j38dAAw4YwIahN9TCmAAEVAADDwTuLQBwFUQUAAMOBMIyAQIKACujoACYMAZE5C4xAOgBwQUAAPPBC6FFAC4CgIKgAFnAoYZFAA9IqAAGHDGdDGDAqBHBBQAA88wgwKgZwQUAAPOsAYFwDUQUAAMOBMgoADoGQEFwMDjQW0AroGAAmDA8ah7ANdCQAEw4EyAB7UB6BkBBcDAY5EsgGsgoAAYcMbwXTwAekZAARBRMUMSZYuJ67FPZ2uLAu1/HaCKAAxGBBQAEZU46mbFJ7h67NPaVK9W//kBqgjAYERAARBZ9hjJZot2FQAGubACSllZmWbMmKHhw4crJSVFCxYsUG1tbUif1tZWFRUVacSIEUpMTFRBQYEaGhpC+tTV1Sk/P18JCQlKSUnR6tWr1dnZ2fezARB1NptdEgEFQN+EFVAqKytVVFSk/fv3q7y8XB0dHZozZ45aWlqCfVatWqW3335b27dvV2Vlpc6dO6cHHngg2N7V1aX8/Hy1t7frww8/1GuvvaYtW7boySefjNxZAYgam90uGzMoAPrIZvrwtKTz588rJSVFlZWVuueee+Tz+TRq1Cht3bpVDz74oCTp5MmTmjRpkqqqqjRz5kzt2rVL9957r86dO6fU1FRJ0ubNm7VmzRqdP39e8fHx1/xcv98vl8sln88np9PZ2/IB9IOW83/Sf+/5pVqb6nvsN/Zb39PIr88coKoAWEE4f7/7tAbF5/NJkpKTkyVJ1dXV6ujoUG5ubrDPxIkTlZmZqaqqKklSVVWVpkyZEgwnkpSXlye/36/jx493+zltbW3y+/0hGwBrstntrEEB0Ge9DiiBQEArV67UXXfdpcmTJ0uSvF6v4uPjlZSUFNI3NTVVXq832Of/hpPL7ZfbulNWViaXyxXcMjIyels2gH5ms8VwiQdAn/U6oBQVFenYsWPatm1bJOvpVmlpqXw+X3A7e/Zsv38mgF6ys0gWQN/F9uZNxcXF2rlzp/bt26fRo0cH97vdbrW3t6upqSlkFqWhoUFutzvY5+DBgyHHu3yXz+U+X+ZwOORwOHpTKoABZrNxmzGAvgtrBsUYo+LiYu3YsUN79uzR2LFjQ9qnTZumuLg4VVRUBPfV1taqrq5OHo9HkuTxeHT06FE1NjYG+5SXl8vpdCorK6sv5wLAAriLB0AkhDWDUlRUpK1bt+qtt97S8OHDg2tGXC6Xhg4dKpfLpSVLlqikpETJyclyOp167LHH5PF4NHPmpdX6c+bMUVZWlhYtWqQNGzbI6/Vq7dq1KioqYpYEuAGwSBZAJIQVUDZt2iRJmjVrVsj+V199VY8++qgk6fnnn5fdbldBQYHa2tqUl5enV155Jdg3JiZGO3fu1IoVK+TxeDRs2DAVFhbq6aef7tuZALAGW4xsrEEB0Ed9eg5KtPAcFMC6Olsv6o+7fq6Wxv/usR/PQQG+egbsOSgAcAU7txkD6DsCCoCIstlYgwKg7wgoACLKZo8Rz0EB0FcEFAARxW3GACKBgAIgwmzXeYnHaBCu0QcwQAgoACLqemdPTKCrnysBMJgRUABEhQkEJGZQAFwFAQVAVFyaQSGgAOgeAQVAVJhAF2tQAFwVAQVAVDCDAqAnBBQAUWECXeQTAFdFQAEQFcygAOgJAQVAVBjDGhQAV0dAARAVJhAQMygAroaAAiAqTFcn+QTAVRFQAESFMQEZEgqAqyCgAIiKS3fxEFAAdI+AAiAquIsHQE8IKACighkUAD0hoACIChMIMH8C4KoIKACighkUAD0hoACIuGEpY2Wzx/TY54vP69TV0TpAFQEYbAgoACIuIXm0bPbYHvu0+RoV6GwfoIoADDYEFAARZ4vpefYEAK6FgAIg4mz2GMkW7SoADGYEFAARd+nyDgkFQO8RUABEnC2m5/UnAHAtBBQAEWe3xzB/AqBPCCgAIu7SGhQiCoDeI6AAiLhrPQMFAK6FgAIg4i7dZswMCoDeCyuglJWVacaMGRo+fLhSUlK0YMEC1dbWhvSZNWuWbDZbyLZ8+fKQPnV1dcrPz1dCQoJSUlK0evVqdXZ29v1sAFiCzcYiWQB9E9a/IpWVlSoqKtKMGTPU2dmpH//4x5ozZ45OnDihYcOGBfstXbpUTz/9dPB1QkJC8Oeuri7l5+fL7Xbrww8/VH19vRYvXqy4uDg988wzETglANFmj+E5KAD6JqyAsnv37pDXW7ZsUUpKiqqrq3XPPfcE9yckJMjtdnd7jN///vc6ceKE3nvvPaWmpur222/XT3/6U61Zs0ZPPfWU4uPje3EaAKyE56AA6Ks+rUHx+XySpOTk5JD9r7/+ukaOHKnJkyertLRUX3zxRbCtqqpKU6ZMUWpqanBfXl6e/H6/jh8/3u3ntLW1ye/3h2wALIxFsgD6qNcXigOBgFauXKm77rpLkydPDu7/zne+ozFjxig9PV1HjhzRmjVrVFtbqzfeeEOS5PV6Q8KJpOBrr9fb7WeVlZVp/fr1vS0VwACz22Nk4zZjAH3Q64BSVFSkY8eO6YMPPgjZv2zZsuDPU6ZMUVpammbPnq3Tp0/r1ltv7dVnlZaWqqSkJPja7/crIyOjd4UD6Hc8SRZAX/XqEk9xcbF27typ999/X6NHj+6xb05OjiTp1KlTkiS3262GhoaQPpdfX23disPhkNPpDNkAWBfPQQHQV2EFFGOMiouLtWPHDu3Zs0djx4695ntqamokSWlpaZIkj8ejo0ePqrGxMdinvLxcTqdTWVlZ4ZQDwKIuBZTruMRjLv27AgBfFtY8bFFRkbZu3aq33npLw4cPD64ZcblcGjp0qE6fPq2tW7dq/vz5GjFihI4cOaJVq1bpnnvuUXZ2tiRpzpw5ysrK0qJFi7RhwwZ5vV6tXbtWRUVFcjgckT9DAAPuetefGNPVz5UAGKzCmkHZtGmTfD6fZs2apbS0tOD261//WpIUHx+v9957T3PmzNHEiRP1+OOPq6CgQG+//XbwGDExMdq5c6diYmLk8Xj0j//4j1q8eHHIc1MAfDWYLh7QCKB7Yc2gXGsqNiMjQ5WVldc8zpgxY/TOO++E89EAbkCBADMoALrHd/EAiBoTYAYFQPcIKACixnQxgwKgewQUAFFjujqiXQIAiyKgAIgawxoUAFdBQAEQNZcWyfIcFABXIqAAiBpuMwZwNQQUAFHDXTwAroaAAiBqTFcXV3gAdIuAAiBqAsygALgKAgqAqGENCoCrIaAAiBpuMwZwNQQUAFET6OoUi1AAdIeAAiBqmEEBcDUEFABRw23GAK6GgAKgX4yY8I1r9rlw6qCMCQxANQAGGwIKgH4Rn+C6Zp/O1ossQQHQLQIKgH5hi4mNdgkABjECCoB+YbcTUAD0HgEFQL9gBgVAX/AvCIBudXV1yZjeLxAxtuv7/09nV6fssvX6c2w2m2JiYnr9fgDWREAB0K3vf//7+uUvf9nr9989JUM/WzZbNtvVw4cxRjcl3aT2zt4/D2X58uV6+eWXe/1+ANZEQAHQrUAgoM7O3j+npLWt47r6dXZ2qrMPASUQ4DZl4EZEQAHQLzr+T+jwdybrQkea2gNDFG9v1Yi4cxoe+79RrA6A1RFQAPSLjq5LMxvn20ertiVHX3Q51aVYxahDCTF+TRq2X8lxf45ylQCsirt4APSL9s4u+TtHqqZ5tpq7RqhLcZJs6lK8mrtG6uPmb6u5MznaZQKwKAIKgH7R2m7Xh00PqNM4um3vNA590FSggLgDB8CVCCgA+kVHV9d1PMW+97cXA7ixEVAA9IuOTu6uAdB7BBQA/aKjD7cOAwABBUC/CHS1K8f1tuzq/lkqdnVqpust2UWQAXClsALKpk2blJ2dLafTKafTKY/Ho127dgXbW1tbVVRUpBEjRigxMVEFBQVqaGgIOUZdXZ3y8/OVkJCglJQUrV69uk8PgwJgTR2dnUqOq9ftwys01O7//0HFyK5ODbX7dYezXDfFNVzzOAC+msJ6Dsro0aP17LPPavz48TLG6LXXXtP999+vw4cP67bbbtOqVav0u9/9Ttu3b5fL5VJxcbEeeOAB/eEPf5B06bs98vPz5Xa79eGHH6q+vl6LFy9WXFycnnnmmX45QQDR0dEZ0Ft/qJVUq6aOj3W+I1NtgQQ57F8oJf5Paor9i2SkLp4EC6AbNtOXbwOTlJycrOeee04PPvigRo0apa1bt+rBBx+UJJ08eVKTJk1SVVWVZs6cqV27dunee+/VuXPnlJqaKknavHmz1qxZo/Pnzys+Pv66PtPv98vlcunRRx+97vcACM++fft08uTJaJdxTVlZWbr77rujXQaA69De3q4tW7bI5/PJ6XT22LfXT5Lt6urS9u3b1dLSIo/Ho+rqanV0dCg3NzfYZ+LEicrMzAwGlKqqKk2ZMiUYTiQpLy9PK1as0PHjx3XHHXd0+1ltbW1qa2sLvvb7/ZKkRYsWKTExsbenAKAHDQ0NgyKgTJo0SUuWLIl2GQCuw8WLF7Vly5br6ht2QDl69Kg8Ho9aW1uVmJioHTt2KCsrSzU1NYqPj1dSUlJI/9TUVHm9XkmS1+sNCSeX2y+3XU1ZWZnWr19/xf7p06dfM4EB6J1Ro0ZFu4TrkpqaqjvvvDPaZQC4DpcnGK5H2HfxTJgwQTU1NTpw4IBWrFihwsJCnThxItzDhKW0tFQ+ny+4nT17tl8/DwAARFfYMyjx8fEaN26cJGnatGk6dOiQXnzxRT300ENqb29XU1NTyCxKQ0OD3G63JMntduvgwYMhx7t8l8/lPt1xOBxyOLp/XDYAALjx9Pk5KIFAQG1tbZo2bZri4uJUUVERbKutrVVdXZ08Ho8kyePx6OjRo2psbAz2KS8vl9PpVFZWVl9LAQAAN4iwZlBKS0s1b948ZWZmqrm5WVu3btXevXv17rvvyuVyacmSJSopKVFycrKcTqcee+wxeTwezZw5U5I0Z84cZWVladGiRdqwYYO8Xq/Wrl2roqIiZkgAAEBQWAGlsbFRixcvVn19vVwul7Kzs/Xuu+/q29/+tiTp+eefl91uV0FBgdra2pSXl6dXXnkl+P6YmBjt3LlTK1askMfj0bBhw1RYWKinn346smcFAAAGtbACyi9+8Yse24cMGaKNGzdq48aNV+0zZswYvfPOO+F8LAAA+Irhu3gAAIDlEFAAAIDlEFAAAIDlEFAAAIDl9Pq7eADc2O644w4tWLAg2mVc09SpU6NdAoB+0OdvM46Gy99mfD3fhggAAKwhnL/fXOIBAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWQ0ABAACWE1ZA2bRpk7Kzs+V0OuV0OuXxeLRr165g+6xZs2Sz2UK25cuXhxyjrq5O+fn5SkhIUEpKilavXq3Ozs7InA0AALghxIbTefTo0Xr22Wc1fvx4GWP02muv6f7779fhw4d12223SZKWLl2qp59+OviehISE4M9dXV3Kz8+X2+3Whx9+qPr6ei1evFhxcXF65plnInRKAABgsLMZY0xfDpCcnKznnntOS5Ys0axZs3T77bfrhRde6Lbvrl27dO+99+rcuXNKTU2VJG3evFlr1qzR+fPnFR8ff12f6ff75XK55PP55HQ6+1I+AAAYIOH8/e71GpSuri5t27ZNLS0t8ng8wf2vv/66Ro4cqcmTJ6u0tFRffPFFsK2qqkpTpkwJhhNJysvLk9/v1/Hjx6/6WW1tbfL7/SEbAAC4cYV1iUeSjh49Ko/Ho9bWViUmJmrHjh3KysqSJH3nO9/RmDFjlJ6eriNHjmjNmjWqra3VG2+8IUnyer0h4URS8LXX673qZ5aVlWn9+vXhlgoAAAapsAPKhAkTVFNTI5/Pp9/+9rcqLCxUZWWlsrKytGzZsmC/KVOmKC0tTbNnz9bp06d166239rrI0tJSlZSUBF/7/X5lZGT0+ngAAMDawr7EEx8fr3HjxmnatGkqKyvT1KlT9eKLL3bbNycnR5J06tQpSZLb7VZDQ0NIn8uv3W73VT/T4XAE7xy6vAEAgBtXn5+DEggE1NbW1m1bTU2NJCktLU2S5PF4dPToUTU2Ngb7lJeXy+l0Bi8TAQAAhHWJp7S0VPPmzVNmZqaam5u1detW7d27V++++65Onz6trVu3av78+RoxYoSOHDmiVatW6Z577lF2drYkac6cOcrKytKiRYu0YcMGeb1erV27VkVFRXI4HP1yggAAYPAJK6A0NjZq8eLFqq+vl8vlUnZ2tt599119+9vf1tmzZ/Xee+/phRdeUEtLizIyMlRQUKC1a9cG3x8TE6OdO3dqxYoV8ng8GjZsmAoLC0OemwIAANDn56BEA89BAQBg8BmQ56AAAAD0FwIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwHAIKAACwnNhoF9AbxhhJkt/vj3IlAADgel3+u33573hPBmVAaW5uliRlZGREuRIAABCu5uZmuVyuHvvYzPXEGIsJBAKqra1VVlaWzp49K6fTGe2SBi2/36+MjAzGMQIYy8hhLCODcYwcxjIyjDFqbm5Wenq67PaeV5kMyhkUu92ur33ta5Ikp9PJL0sEMI6Rw1hGDmMZGYxj5DCWfXetmZPLWCQLAAAsh4ACAAAsZ9AGFIfDoXXr1snhcES7lEGNcYwcxjJyGMvIYBwjh7EceINykSwAALixDdoZFAAAcOMioAAAAMshoAAAAMshoAAAAMsZlAFl48aNuvnmmzVkyBDl5OTo4MGD0S7Jcvbt26f77rtP6enpstlsevPNN0PajTF68sknlZaWpqFDhyo3N1effvppSJ8LFy5o4cKFcjqdSkpK0pIlS3Tx4sUBPIvoKysr04wZMzR8+HClpKRowYIFqq2tDenT2tqqoqIijRgxQomJiSooKFBDQ0NIn7q6OuXn5yshIUEpKSlavXq1Ojs7B/JUomrTpk3Kzs4OPuTK4/Fo165dwXbGsPeeffZZ2Ww2rVy5MriP8bw+Tz31lGw2W8g2ceLEYDvjGGVmkNm2bZuJj483v/zlL83x48fN0qVLTVJSkmloaIh2aZbyzjvvmH/6p38yb7zxhpFkduzYEdL+7LPPGpfLZd58803zX//1X+bv//7vzdixY81f//rXYJ+5c+eaqVOnmv3795v//M//NOPGjTOPPPLIAJ9JdOXl5ZlXX33VHDt2zNTU1Jj58+ebzMxMc/HixWCf5cuXm4yMDFNRUWE++ugjM3PmTPONb3wj2N7Z2WkmT55scnNzzeHDh80777xjRo4caUpLS6NxSlHxH//xH+Z3v/ud+eMf/2hqa2vNj3/8YxMXF2eOHTtmjGEMe+vgwYPm5ptvNtnZ2eYHP/hBcD/jeX3WrVtnbrvtNlNfXx/czp8/H2xnHKNr0AWUO++80xQVFQVfd3V1mfT0dFNWVhbFqqztywElEAgYt9ttnnvuueC+pqYm43A4zK9+9StjjDEnTpwwksyhQ4eCfXbt2mVsNpv585//PGC1W01jY6ORZCorK40xl8YtLi7ObN++Pdjnk08+MZJMVVWVMeZSWLTb7cbr9Qb7bNq0yTidTtPW1jawJ2AhN910k/m3f/s3xrCXmpubzfjx4015ebn527/922BAYTyv37p168zUqVO7bWMco29QXeJpb29XdXW1cnNzg/vsdrtyc3NVVVUVxcoGlzNnzsjr9YaMo8vlUk5OTnAcq6qqlJSUpOnTpwf75Obmym6368CBAwNes1X4fD5JUnJysiSpurpaHR0dIWM5ceJEZWZmhozllClTlJqaGuyTl5cnv9+v48ePD2D11tDV1aVt27appaVFHo+HMeyloqIi5efnh4ybxO9kuD799FOlp6frlltu0cKFC1VXVyeJcbSCQfVlgX/5y1/U1dUV8ssgSampqTp58mSUqhp8vF6vJHU7jpfbvF6vUlJSQtpjY2OVnJwc7PNVEwgEtHLlSt11112aPHmypEvjFB8fr6SkpJC+Xx7L7sb6cttXxdGjR+XxeNTa2qrExETt2LFDWVlZqqmpYQzDtG3bNn388cc6dOjQFW38Tl6/nJwcbdmyRRMmTFB9fb3Wr1+vb37zmzp27BjjaAGDKqAA0VRUVKRjx47pgw8+iHYpg9KECRNUU1Mjn8+n3/72tyosLFRlZWW0yxp0zp49qx/84AcqLy/XkCFDol3OoDZv3rzgz9nZ2crJydGYMWP0m9/8RkOHDo1iZZAG2V08I0eOVExMzBWrqBsaGuR2u6NU1eBzeax6Gke3263GxsaQ9s7OTl24cOErOdbFxcXauXOn3n//fY0ePTq43+12q729XU1NTSH9vzyW3Y315bavivj4eI0bN07Tpk1TWVmZpk6dqhdffJExDFN1dbUaGxv1N3/zN4qNjVVsbKwqKyv10ksvKTY2VqmpqYxnLyUlJenrX/+6Tp06xe+lBQyqgBIfH69p06apoqIiuC8QCKiiokIejyeKlQ0uY8eOldvtDhlHv9+vAwcOBMfR4/GoqalJ1dXVwT579uxRIBBQTk7OgNccLcYYFRcXa8eOHdqzZ4/Gjh0b0j5t2jTFxcWFjGVtba3q6upCxvLo0aMhga+8vFxOp1NZWVkDcyIWFAgE1NbWxhiGafbs2Tp69KhqamqC2/Tp07Vw4cLgz4xn71y8eFGnT59WWloav5dWEO1VuuHatm2bcTgcZsuWLebEiRNm2bJlJikpKWQVNS6t8D98+LA5fPiwkWT+9V//1Rw+fNj86U9/MsZcus04KSnJvPXWW+bIkSPm/vvv7/Y24zvuuMMcOHDAfPDBB2b8+PFfuduMV6xYYVwul9m7d2/IrYhffPFFsM/y5ctNZmam2bNnj/noo4+Mx+MxHo8n2H75VsQ5c+aYmpoas3v3bjNq1Kiv1K2ITzzxhKmsrDRnzpwxR44cMU888YSx2Wzm97//vTGGMeyr/3sXjzGM5/V6/PHHzd69e82ZM2fMH/7wB5Obm2tGjhxpGhsbjTGMY7QNuoBijDEvv/yyyczMNPHx8ebOO+80+/fvj3ZJlvP+++8bSVdshYWFxphLtxr/5Cc/MampqcbhcJjZs2eb2trakGN8/vnn5pFHHjGJiYnG6XSa7373u6a5uTkKZxM93Y2hJPPqq68G+/z1r3813//+981NN91kEhISzD/8wz+Y+vr6kOP8z//8j5k3b54ZOnSoGTlypHn88cdNR0fHAJ9N9Hzve98zY8aMMfHx8WbUqFFm9uzZwXBiDGPYV18OKIzn9XnooYdMWlqaiY+PN1/72tfMQw89ZE6dOhVsZxyjy2aMMdGZuwEAAOjeoFqDAgAAvhoIKAAAwHIIKAAAwHIIKAAAwHIIKAAAwHIIKAAAwHIIKAAAwHIIKAAAwHIIKAAAwHIIKAAAwHIIKAAAwHIIKAAAwHL+H8ak+rmhU4A7AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Gym",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
